#code:utf-8
import datetime
import time
import csv

from bs4 import BeautifulSoup
import json
from pymongo import MongoClient
import urllib.parse
import re
import sys   
from MongoTiebaListQueue import MongoTiebaListQueue
import threading
import math
sys.setrecursionlimit(1000000) #例如这里设置为一百万 
baseurl='http://tieba.baidu.com/bawu2/platform/listMemberInfo?word={tiebaname}&ie=utf-8&pn={pagenumber}'
#baseurl='http://tieba.baidu.com'


client=MongoClient('192.168.0.124',27017)
db=client.cache


if __name__ == '__main__':    
    # threaded_crawler(10)
    # x=0
    # for i in db.proxy.find({'status':2}):
    #     x=x+1
    #     print(x)
    #     db.proxy.update({'_id':i['_id']},{'$set':{'status':0}})
    # for i in db.proxy.find({'status':{'$ne':2}}):
    #     x=x+1
    #     print(x)
    #     db.proxy.remove({'_id':i['_id']})
    # for i in db.tiebaurl2.find():        
    #    db.tiebaurl2.update({'_id':i['_id']},{'$set':{'status':0}})
    # for i in db.tieba.find({'status':{'$ne':2}}):        
    #    db.tieba.update({'_id':i['_id']},{'$set':{'status':0}})
    # for i in dbcache.tiabuser1.find({'status1':1}):        
    #    dbcache.tiabuser1.update({'_id':i['_id']},{'$set':{'status1':0}})
    # if dbcache.orglist.find({'status3':0}).count()==0:
    #     for i in dbcache.orglist.find():        
    #         dbcache.orglist.update({'_id':i['_id']},{'$set':{'status3':0}})
    # else:
    #     print('no update')
    
    # with open("colname.csv","w",newline="",encoding="utf-8") as csvfile:    
    #     writer = csv.writer(csvfile,dialect = ("excel"))
    #     writer.writerow(['列'])
    # for i in dbcache.tiabuser1.find():
    #     for x in i:
    #         dic[x]=''
    # #     print(dic)
#url='http://tieba.baidu.com/f/index/forumclass'
    
    # for i in dbcache.tiabuser1.find():
    #     for key,value in dic.items():
    #         if key in i.keys():
    #             dic[key]=i[key]
    #         else:
    #             dic[key]=''

    #     #print(dic)
    #     dbcache.tiabuser1.update({'_id':i['_id']},{'$set':dic})

    # basennurl='http://www.xicidaili.com/nn/'
    # basewnurl='http://www.xicidaili.com/wn/'
    # for i in range(1,10):
    #     db.urllist.update({'_id':basennurl+str(i)},{'$set':{'status':0}},upsert=True)
    #     db.urllist.update({'_id':basewnurl+str(i)},{'$set':{'status':0}},upsert=True)




    j=0    
    while j<40000:

        j=j+1
        print(j)
        i=db.tiebaurl.find().limit(100)
        newi=0
        newi=i
        fields=[]
        js=[]
        
        for x in i:
            #print(x['_id'])
            fields.append(x['_id'])
            dic={}
            dic['_id']=x['_id']
            dic['tieba']=x['tieba']
            dic['status']=x['status']
            dic['html']=x['html']
            dic['num']=x['num']
            js.append(dic)
        
        db.tiebaurl01.insert_many(js)        
        db.tiebaurl.remove({'_id':{'$in':fields}})




    #for i in db.tiebaurl.find():
    # while i:
    #     x=x+1
    #     print(x)
    #     if x>10000000:
    #         break
    #     else:
    #         db.tiebaurl.remove({'_id':i['_id']})
    #         db.tiebaurl2.insert({'_id':i['_id'],'tieba':i['tieba'],'status':i['status'],'html':i['html'],'num':i['num']})
            
    #         i=db.tiebaurl.find_one()
    # for i in db.tiebaurl.find({'status':3}):        
    #     db.tiebaurl.update({'_id':i['_id']},{'$set':{'status':0,'type':i['type'],'pnum':i['pnum'],'desc':i['desc'],'img':i['img']}},upsert=True)